import scrapy
from selenium import webdriver
from WangyiNews.items import WangyinewsItem


class NewsSpider(scrapy.Spider):
    name = 'news'
    # allowed_domains = ['www.bbb.com']
    start_urls = ['https://news.163.com/']
    # 每个新闻模块的url
    model_urls = []
    
    def __init__(self):
        self.driver = webdriver.Chrome(executable_path='E:\\python\程序\\net_spyder\\chromedriver.exe')
        self.driver.execute_cdp_cmd("Page.addScriptToEvaluateOnNewDocument", {
            "source": """
                Object.defineProperty(navigator, 'webdriver', {
                get: () => undefined
                })
            """
            })
        
    def parse(self, response):
        """获取模块地址"""
        li_list = response.xpath('//*[@id="index2016_wrap"]/div[1]/div[2]/div[2]/div[2]/div[2]/div/ul/li')
        for li in li_list[3:5]:
            self.model_urls.append(li.xpath('./a/@href').extract_first())
        print(self.model_urls)
        for url in self.model_urls:
            yield scrapy.Request(url, callback=self.parse_model)
            
    def parse_model(self, response):
        """解析模块中的新闻标题和地址"""
        # 这里response经过中间件处理
        for div in response.xpath('/html/body/div/div[3]/div[4]/div[1]/div[1]/div/ul/li/div/div'):
            title = div.xpath('.//div[@class="news_title"]/h3/a/text()')[0].extract()
            url = div.xpath('.//div[@class="news_title"]/h3/a/@href')[0].extract()
            # print("title", title)
            item = WangyinewsItem()
            item["title"] = title
            
            yield scrapy.Request(url, callback=self.parse_content, meta={'item': item})
        
    def parse_content(self, response):
        content = response.xpath('//*[@id="content"]/div[2]//text()').extract()
        content = ''.join(content)
        item = response.meta['item']
        item['content'] = content
        
        yield item
        
    def closed(self, spider):
        self.driver.quit()